/************************************************************************* > File Name: boundarympi.c
  > Author: wusihai
  > Mail: wusihai18@gmail.com 
  > Created Time: 三  1/29 12:52:25 2020
 ************************************************************************/
#include "rtmlib.h" 
#include "slavertm.h"
#include <athread.h> /*
					  * 1.申请存储，把x,y,z方向的数据分别打包-可考虑从核函数,在两个函数实现T,V的打包
					  * 2.启动三个异步通信函数,分别通信T和V
					  * 3.io也打包。
					  * 4.打包函数并行性好，可作为独立kernel，不用重叠计算。
					  * 4.重叠部分，仅包含通信和读写部分。
					  */
//#define MCPY
/*正传-反传-通信V-Y-Z方向*/

/*
 * mpi_pack函数 ,v,t通用
 */
extern SLAVE_FUN(SmpipackX)(rtm_slave_mpipack* p);
extern SLAVE_FUN(SmpipackY)(rtm_slave_mpipack* p);
extern SLAVE_FUN(SmpipackZ)(rtm_slave_mpipack* p);
extern SLAVE_FUN(SmpiunpackX)(rtm_slave_mpipack* p);
extern SLAVE_FUN(SmpiunpackY)(rtm_slave_mpipack* p);
extern SLAVE_FUN(SmpiunpackZ)(rtm_slave_mpipack* p);
void MPI_LaunchV(){

  int i,j,k;
  parameter_mpipack.Pin[0]=Vx;
  parameter_mpipack.Pin[1]=Vy;
  parameter_mpipack.Pin[2]=Vz;
  parameter_mpipack.Pin[3]=Vx;
  parameter_mpipack.Pin[4]=Vy;
  parameter_mpipack.Pin[5]=Vz;

  /*1. X-方向*/
  int size=5*DivNy*DivNz;
  P1=sendX0;
  P2=sendX0+1*size;
  P3=sendX0+2*size;
  P4=sendX0+3*size;
  P5=sendX0+4*size;
  P6=sendX0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackX,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendX0 ,size*3,MPI_FLOAT,nbrs[X0],0,MPI_COMM_WORLD,&reqs[0]);
  MPI_Irecv(recvX0 ,size*3,MPI_FLOAT,nbrs[X1],0,MPI_COMM_WORLD,&reqs[1]);
  MPI_Isend(sendX0+size*3,size*3,MPI_FLOAT,nbrs[X1],1,MPI_COMM_WORLD,&reqs[2]);
  MPI_Irecv(recvX0+size*3,size*3,MPI_FLOAT,nbrs[X0],1,MPI_COMM_WORLD,&reqs[3]);

  /*2.Y 方向*/
  size=DivNx*5*DivNz;
  P1=sendY0;
  P2=sendY0+1*size;
  P3=sendY0+2*size;
  P4=sendY0+3*size;
  P5=sendY0+4*size;
  P6=sendY0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackY,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendY0       ,size*3,MPI_FLOAT,nbrs[Y0],2,MPI_COMM_WORLD,&reqs[4]);
  MPI_Irecv(recvY0       ,size*3,MPI_FLOAT,nbrs[Y1],2,MPI_COMM_WORLD,&reqs[5]);
  MPI_Isend(sendY0+size*3,size*3,MPI_FLOAT,nbrs[Y1],3,MPI_COMM_WORLD,&reqs[6]);
  MPI_Irecv(recvY0+size*3,size*3,MPI_FLOAT,nbrs[Y0],3,MPI_COMM_WORLD,&reqs[7]);

  /*3. Z方向 */
  size=DivNx*DivNy*5;
  P1=sendZ0;
  P2=sendZ0+1*size;
  P3=sendZ0+2*size;
  P4=sendZ0+3*size;
  P5=sendZ0+4*size;
  P6=sendZ0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;


  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  athread_spawn(SmpipackZ,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendZ0       ,size*3,MPI_FLOAT,nbrs[Z0],4,MPI_COMM_WORLD,&reqs[8]);
  MPI_Irecv(recvZ0       ,size*3,MPI_FLOAT,nbrs[Z1],4,MPI_COMM_WORLD,&reqs[9]);
  MPI_Isend(sendZ0+size*3,size*3,MPI_FLOAT,nbrs[Z1],5,MPI_COMM_WORLD,&reqs[10]);
  MPI_Irecv(recvZ0+size*3,size*3,MPI_FLOAT,nbrs[Z0],5,MPI_COMM_WORLD,&reqs[11]);

}
void MPI_JoinV(){
  int i,j,k;
  parameter_mpipack.Pout[0]=Vx;
  parameter_mpipack.Pout[1]=Vy;
  parameter_mpipack.Pout[2]=Vz;
  parameter_mpipack.Pout[3]=Vx;
  parameter_mpipack.Pout[4]=Vy;
  parameter_mpipack.Pout[5]=Vz;

  /*1. X-方向*/
  MPI_Wait(&reqs[0],&status);
  MPI_Wait(&reqs[1],&status);

  int size=5*DivNy*DivNz;
  P1=recvX0;
  P2=recvX0+1*size;
  P3=recvX0+2*size;
  P4=recvX0+3*size;
  P5=recvX0+4*size;
  P6=recvX0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;

  if(x_rank<dims[0]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[2],&status);
  MPI_Wait(&reqs[3],&status);
  if(x_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }

  /*2.Y-方向*/
  MPI_Wait(&reqs[4],&status);
  MPI_Wait(&reqs[5],&status);
  size=DivNx*5*DivNz;
  P1=recvY0;
  P2=recvY0+1*size;
  P3=recvY0+2*size;
  P4=recvY0+3*size;
  P5=recvY0+4*size;
  P6=recvY0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;

  if(y_rank<dims[1]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[6],&status);
  MPI_Wait(&reqs[7],&status);
  if(y_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }
  /*3.Z-方向*/
  MPI_Wait(&reqs[8],&status);
  MPI_Wait(&reqs[9],&status);
  size=DivNx*DivNy*5;
  P1=recvZ0;
  P2=recvZ0+1*size;
  P3=recvZ0+2*size;
  P4=recvZ0+3*size;
  P5=recvZ0+4*size;
  P6=recvZ0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  if(z_rank<dims[2]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[10],&status);
  MPI_Wait(&reqs[11],&status);
  if(z_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
}

void MPI_LaunchT(){
  int i,j,k;
  parameter_mpipack.Pin[0]=Txx;
  parameter_mpipack.Pin[1]=Tyy;
  parameter_mpipack.Pin[2]=Tzz;
  parameter_mpipack.Pin[3]=Txx;
  parameter_mpipack.Pin[4]=Tyy;
  parameter_mpipack.Pin[5]=Tzz;
  /*1. X-方向*/
  int size=5*DivNy*DivNz;
  P1=sendX0;
  P2=sendX0+1*size;
  P3=sendX0+2*size;
  P4=sendX0+3*size;
  P5=sendX0+4*size;
  P6=sendX0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackX,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendX0 ,size*3,MPI_FLOAT,nbrs[X0],0,MPI_COMM_WORLD,&reqs[0]);
  MPI_Irecv(recvX0 ,size*3,MPI_FLOAT,nbrs[X1],0,MPI_COMM_WORLD,&reqs[1]);
  MPI_Isend(sendX0+size*3,size*3,MPI_FLOAT,nbrs[X1],1,MPI_COMM_WORLD,&reqs[2]);
  MPI_Irecv(recvX0+size*3,size*3,MPI_FLOAT,nbrs[X0],1,MPI_COMM_WORLD,&reqs[3]);


  /*2.Y 方向*/
  size=DivNx*5*DivNz;
  P1=sendY0;
  P2=sendY0+1*size;
  P3=sendY0+2*size;
  P4=sendY0+3*size;
  P5=sendY0+4*size;
  P6=sendY0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackY,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendY0       ,size*3,MPI_FLOAT,nbrs[Y0],2,MPI_COMM_WORLD,&reqs[4]);
  MPI_Irecv(recvY0       ,size*3,MPI_FLOAT,nbrs[Y1],2,MPI_COMM_WORLD,&reqs[5]);
  MPI_Isend(sendY0+size*3,size*3,MPI_FLOAT,nbrs[Y1],3,MPI_COMM_WORLD,&reqs[6]);
  MPI_Irecv(recvY0+size*3,size*3,MPI_FLOAT,nbrs[Y0],3,MPI_COMM_WORLD,&reqs[7]);

  /*3. Z方向 */
  size=DivNx*DivNy*5;
  P1=sendZ0;
  P2=sendZ0+1*size;
  P3=sendZ0+2*size;
  P4=sendZ0+3*size;
  P5=sendZ0+4*size;
  P6=sendZ0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  athread_spawn(SmpipackZ,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendZ0       ,size*3,MPI_FLOAT,nbrs[Z0],4,MPI_COMM_WORLD,&reqs[8]);
  MPI_Irecv(recvZ0       ,size*3,MPI_FLOAT,nbrs[Z1],4,MPI_COMM_WORLD,&reqs[9]);
  MPI_Isend(sendZ0+size*3,size*3,MPI_FLOAT,nbrs[Z1],5,MPI_COMM_WORLD,&reqs[10]);
  MPI_Irecv(recvZ0+size*3,size*3,MPI_FLOAT,nbrs[Z0],5,MPI_COMM_WORLD,&reqs[11]);
}
void MPI_JoinT(){
  int i,j,k;
  parameter_mpipack.Pout[0]=Txx;
  parameter_mpipack.Pout[1]=Tyy;
  parameter_mpipack.Pout[2]=Tzz;
  parameter_mpipack.Pout[3]=Txx;
  parameter_mpipack.Pout[4]=Tyy;
  parameter_mpipack.Pout[5]=Tzz;

  /*1. X-方向*/
  MPI_Wait(&reqs[0],&status);
  MPI_Wait(&reqs[1],&status);

  int size=5*DivNy*DivNz;
  P1=recvX0;
  P2=recvX0+1*size;
  P3=recvX0+2*size;
  P4=recvX0+3*size;
  P5=recvX0+4*size;
  P6=recvX0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;


  if(x_rank<dims[0]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[2],&status);
  MPI_Wait(&reqs[3],&status);
  if(x_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }
  /*2.Y-方向*/
  MPI_Wait(&reqs[4],&status);
  MPI_Wait(&reqs[5],&status);
  size=DivNx*5*DivNz;
  P1=recvY0;
  P2=recvY0+1*size;
  P3=recvY0+2*size;
  P4=recvY0+3*size;
  P5=recvY0+4*size;
  P6=recvY0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;

  if(y_rank<dims[1]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[6],&status);
  MPI_Wait(&reqs[7],&status);
  if(y_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }

  /*3.Z-方向*/
  MPI_Wait(&reqs[8],&status);
  MPI_Wait(&reqs[9],&status);
  size=DivNx*DivNy*5;
  P1=recvZ0;
  P2=recvZ0+1*size;
  P3=recvZ0+2*size;
  P4=recvZ0+3*size;
  P5=recvZ0+4*size;
  P6=recvZ0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  if(z_rank<dims[2]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[10],&status);
  MPI_Wait(&reqs[11],&status);
  if(z_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
}

/*
 *
 *
 */
void MPI_LaunchV_extrap(){

  int i,j,k;
  parameter_mpipack.Pin[0]=extrapVx;
  parameter_mpipack.Pin[1]=extrapVy;
  parameter_mpipack.Pin[2]=extrapVz;
  parameter_mpipack.Pin[3]=extrapVx;
  parameter_mpipack.Pin[4]=extrapVy;
  parameter_mpipack.Pin[5]=extrapVz;

  /*1. X-方向*/
  int size=5*DivNy*DivNz;
  P1=sendX0;
  P2=sendX0+1*size;
  P3=sendX0+2*size;
  P4=sendX0+3*size;
  P5=sendX0+4*size;
  P6=sendX0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackX,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendX0 ,size*3,MPI_FLOAT,nbrs[X0],0,MPI_COMM_WORLD,&reqs[0]);
  MPI_Irecv(recvX0 ,size*3,MPI_FLOAT,nbrs[X1],0,MPI_COMM_WORLD,&reqs[1]);
  MPI_Isend(sendX0+size*3,size*3,MPI_FLOAT,nbrs[X1],1,MPI_COMM_WORLD,&reqs[2]);
  MPI_Irecv(recvX0+size*3,size*3,MPI_FLOAT,nbrs[X0],1,MPI_COMM_WORLD,&reqs[3]);

  /*2.Y 方向*/
  size=DivNx*5*DivNz;
  P1=sendY0;
  P2=sendY0+1*size;
  P3=sendY0+2*size;
  P4=sendY0+3*size;
  P5=sendY0+4*size;
  P6=sendY0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackY,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendY0       ,size*3,MPI_FLOAT,nbrs[Y0],2,MPI_COMM_WORLD,&reqs[4]);
  MPI_Irecv(recvY0       ,size*3,MPI_FLOAT,nbrs[Y1],2,MPI_COMM_WORLD,&reqs[5]);
  MPI_Isend(sendY0+size*3,size*3,MPI_FLOAT,nbrs[Y1],3,MPI_COMM_WORLD,&reqs[6]);
  MPI_Irecv(recvY0+size*3,size*3,MPI_FLOAT,nbrs[Y0],3,MPI_COMM_WORLD,&reqs[7]);

  /*3. Z方向 */
  size=DivNx*DivNy*5;
  P1=sendZ0;
  P2=sendZ0+1*size;
  P3=sendZ0+2*size;
  P4=sendZ0+3*size;
  P5=sendZ0+4*size;
  P6=sendZ0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;


  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  athread_spawn(SmpipackZ,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendZ0       ,size*3,MPI_FLOAT,nbrs[Z0],4,MPI_COMM_WORLD,&reqs[8]);
  MPI_Irecv(recvZ0       ,size*3,MPI_FLOAT,nbrs[Z1],4,MPI_COMM_WORLD,&reqs[9]);
  MPI_Isend(sendZ0+size*3,size*3,MPI_FLOAT,nbrs[Z1],5,MPI_COMM_WORLD,&reqs[10]);
  MPI_Irecv(recvZ0+size*3,size*3,MPI_FLOAT,nbrs[Z0],5,MPI_COMM_WORLD,&reqs[11]);
}
void MPI_JoinV_extrap(){
  int i,j,k;
  parameter_mpipack.Pout[0]=extrapVx;
  parameter_mpipack.Pout[1]=extrapVy;
  parameter_mpipack.Pout[2]=extrapVz;
  parameter_mpipack.Pout[3]=extrapVx;
  parameter_mpipack.Pout[4]=extrapVy;
  parameter_mpipack.Pout[5]=extrapVz;

  /*1. X-方向*/
  MPI_Wait(&reqs[0],&status);
  MPI_Wait(&reqs[1],&status);

  int size=5*DivNy*DivNz;
  P1=recvX0;
  P2=recvX0+1*size;
  P3=recvX0+2*size;
  P4=recvX0+3*size;
  P5=recvX0+4*size;
  P6=recvX0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;

  if(x_rank<dims[0]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[2],&status);
  MPI_Wait(&reqs[3],&status);
  if(x_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }

  /*2.Y-方向*/
  MPI_Wait(&reqs[4],&status);
  MPI_Wait(&reqs[5],&status);
  size=DivNx*5*DivNz;
  P1=recvY0;
  P2=recvY0+1*size;
  P3=recvY0+2*size;
  P4=recvY0+3*size;
  P5=recvY0+4*size;
  P6=recvY0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;

  if(y_rank<dims[1]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[6],&status);
  MPI_Wait(&reqs[7],&status);
  if(y_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }

  /*3.Z-方向*/
  MPI_Wait(&reqs[8],&status);
  MPI_Wait(&reqs[9],&status);
  size=DivNx*DivNy*5;
  P1=recvZ0;
  P2=recvZ0+1*size;
  P3=recvZ0+2*size;
  P4=recvZ0+3*size;
  P5=recvZ0+4*size;
  P6=recvZ0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  if(z_rank<dims[2]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[10],&status);
  MPI_Wait(&reqs[11],&status);
  if(z_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
}

void MPI_LaunchT_extrap(){
  int i,j,k;
  parameter_mpipack.Pin[0]=extrapTxx;
  parameter_mpipack.Pin[1]=extrapTyy;
  parameter_mpipack.Pin[2]=extrapTzz;
  parameter_mpipack.Pin[3]=extrapTxx;
  parameter_mpipack.Pin[4]=extrapTyy;
  parameter_mpipack.Pin[5]=extrapTzz;
  /*1. X-方向*/
  int size=5*DivNy*DivNz;
  P1=sendX0;
  P2=sendX0+1*size;
  P3=sendX0+2*size;
  P4=sendX0+3*size;
  P5=sendX0+4*size;
  P6=sendX0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackX,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendX0 ,size*3,MPI_FLOAT,nbrs[X0],0,MPI_COMM_WORLD,&reqs[0]);
  MPI_Irecv(recvX0 ,size*3,MPI_FLOAT,nbrs[X1],0,MPI_COMM_WORLD,&reqs[1]);
  MPI_Isend(sendX0+size*3,size*3,MPI_FLOAT,nbrs[X1],1,MPI_COMM_WORLD,&reqs[2]);
  MPI_Irecv(recvX0+size*3,size*3,MPI_FLOAT,nbrs[X0],1,MPI_COMM_WORLD,&reqs[3]);


  /*2.Y 方向*/
  size=DivNx*5*DivNz;
  P1=sendY0;
  P2=sendY0+1*size;
  P3=sendY0+2*size;
  P4=sendY0+3*size;
  P5=sendY0+4*size;
  P6=sendY0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;
  athread_spawn(SmpipackY,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendY0       ,size*3,MPI_FLOAT,nbrs[Y0],2,MPI_COMM_WORLD,&reqs[4]);
  MPI_Irecv(recvY0       ,size*3,MPI_FLOAT,nbrs[Y1],2,MPI_COMM_WORLD,&reqs[5]);
  MPI_Isend(sendY0+size*3,size*3,MPI_FLOAT,nbrs[Y1],3,MPI_COMM_WORLD,&reqs[6]);
  MPI_Irecv(recvY0+size*3,size*3,MPI_FLOAT,nbrs[Y0],3,MPI_COMM_WORLD,&reqs[7]);

  /*3. Z方向 */
  size=DivNx*DivNy*5;
  P1=sendZ0;
  P2=sendZ0+1*size;
  P3=sendZ0+2*size;
  P4=sendZ0+3*size;
  P5=sendZ0+4*size;
  P6=sendZ0+5*size;

  parameter_mpipack.Pout[0]=P1;
  parameter_mpipack.Pout[1]=P2;
  parameter_mpipack.Pout[2]=P3;
  parameter_mpipack.Pout[3]=P4;
  parameter_mpipack.Pout[4]=P5;
  parameter_mpipack.Pout[5]=P6;

  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  athread_spawn(SmpipackZ,&parameter_mpipack);
  athread_join();

  MPI_Isend(sendZ0       ,size*3,MPI_FLOAT,nbrs[Z0],4,MPI_COMM_WORLD,&reqs[8]);
  MPI_Irecv(recvZ0       ,size*3,MPI_FLOAT,nbrs[Z1],4,MPI_COMM_WORLD,&reqs[9]);
  MPI_Isend(sendZ0+size*3,size*3,MPI_FLOAT,nbrs[Z1],5,MPI_COMM_WORLD,&reqs[10]);
  MPI_Irecv(recvZ0+size*3,size*3,MPI_FLOAT,nbrs[Z0],5,MPI_COMM_WORLD,&reqs[11]);
}
void MPI_JoinT_extrap(){
  int i,j,k;
  parameter_mpipack.Pout[0]=extrapTxx;
  parameter_mpipack.Pout[1]=extrapTyy;
  parameter_mpipack.Pout[2]=extrapTzz;
  parameter_mpipack.Pout[3]=extrapTxx;
  parameter_mpipack.Pout[4]=extrapTyy;
  parameter_mpipack.Pout[5]=extrapTzz;

  /*1. X-方向*/
  MPI_Wait(&reqs[0],&status);
  MPI_Wait(&reqs[1],&status);

  int size=5*DivNy*DivNz;
  P1=recvX0;
  P2=recvX0+1*size;
  P3=recvX0+2*size;
  P4=recvX0+3*size;
  P5=recvX0+4*size;
  P6=recvX0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=5;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;


  if(x_rank<dims[0]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[2],&status);
  MPI_Wait(&reqs[3],&status);
  if(x_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackX,&parameter_mpipack);
	athread_join();
  }
  /*2.Y-方向*/
  MPI_Wait(&reqs[4],&status);
  MPI_Wait(&reqs[5],&status);
  size=DivNx*5*DivNz;
  P1=recvY0;
  P2=recvY0+1*size;
  P3=recvY0+2*size;
  P4=recvY0+3*size;
  P5=recvY0+4*size;
  P6=recvY0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=5;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=DivNz;

  if(y_rank<dims[1]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[6],&status);
  MPI_Wait(&reqs[7],&status);
  if(y_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackY,&parameter_mpipack);
	athread_join();
  }

  /*3.Z-方向*/
  MPI_Wait(&reqs[8],&status);
  MPI_Wait(&reqs[9],&status);
  size=DivNx*DivNy*5;
  P1=recvZ0;
  P2=recvZ0+1*size;
  P3=recvZ0+2*size;
  P4=recvZ0+3*size;
  P5=recvZ0+4*size;
  P6=recvZ0+5*size;
  parameter_mpipack.Pin[0]=P1;
  parameter_mpipack.Pin[1]=P2;
  parameter_mpipack.Pin[2]=P3;
  parameter_mpipack.Pin[3]=P4;
  parameter_mpipack.Pin[4]=P5;
  parameter_mpipack.Pin[5]=P6;
  parameter_mpipack.stX =0;
  parameter_mpipack.endX=DivNx;
  parameter_mpipack.stY =0;
  parameter_mpipack.endY=DivNy;
  parameter_mpipack.stZ =0;
  parameter_mpipack.endZ=5;
  if(z_rank<dims[2]-1){
	parameter_mpipack.option=0;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
  MPI_Wait(&reqs[10],&status);
  MPI_Wait(&reqs[11],&status);
  if(z_rank>0){
	parameter_mpipack.option=1;
	athread_spawn(SmpiunpackZ,&parameter_mpipack);
	athread_join();
  }
}

/*
 *
 *
 *
 */
